{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/chatbot/blended-skill-talk/blended_skill_talk.json.translate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def cleaning(string):\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('blended_skill_talk.json.translate') as fopen:\n",
    "    data = json.load(fopen)\n",
    "\n",
    "before, after = [], []\n",
    "\n",
    "for i in data:\n",
    "    if len(i) != 2:\n",
    "        continue\n",
    "    text = i[1]\n",
    "    splitted = text.split('[EENND]')\n",
    "    if len(splitted) != 2:\n",
    "        continue\n",
    "    l, r = splitted\n",
    "    l = cleaning(l)\n",
    "    r = cleaning(r)\n",
    "    before.append(l)\n",
    "    after.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/chatbot/convai2/convai2-0.json.translate\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/chatbot/convai2/convai2-100000.json.translate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31877, 31877)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(before), len(after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('convai2-0.json.translate') as fopen:\n",
    "    data = json.load(fopen)\n",
    "\n",
    "for i in data:\n",
    "    if len(i) != 2:\n",
    "        continue\n",
    "    text = i[1]\n",
    "    splitted = text.split('[EENND]')\n",
    "    if len(splitted) == 3:\n",
    "        splitted = splitted[1:]\n",
    "    if len(splitted) != 2:\n",
    "        continue\n",
    "    l, r = splitted\n",
    "    l = cleaning(l)\n",
    "    r = cleaning(r)\n",
    "    before.append(l)\n",
    "    after.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('convai2-100000.json.translate') as fopen:\n",
    "    data = json.load(fopen)\n",
    "\n",
    "for i in data:\n",
    "    if len(i) != 2:\n",
    "        continue\n",
    "    text = i[1]\n",
    "    splitted = text.split('[EENND]')\n",
    "    if len(splitted) == 3:\n",
    "        splitted = splitted[1:]\n",
    "    if len(splitted) != 2:\n",
    "        continue\n",
    "    l, r = splitted\n",
    "    l = cleaning(l)\n",
    "    r = cleaning(r)\n",
    "    before.append(l)\n",
    "    after.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(142685, 142685)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(before), len(after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Hai apa khabar?', 'Saya hebat pada malam musim gugur ini'),\n",
       " ('Adakah itu orang yang anda harapkan.',\n",
       "  'Ya, dia yang terbaik. Adakah anda fikir begitu?'),\n",
       " ('Adakah anda suka sukan? Saya suka tenis dan bermain untuk hidup.',\n",
       "  'Sukan tidak baik tetapi saya suka pertunjukan memasak dan memasak'),\n",
       " ('Pergi ke kedai kopi.',\n",
       "  'Saya kadang-kadang memainkan lagu-lagu rakyat di kedai kopi. Ia adalah baik.'),\n",
       " ('Suami saya seorang doktor jadi saya memasak makanan dan membekukannya.',\n",
       "  'Itu adalah idea hebat yang saya lajang, jadi saya cukup untuk satu sahaja'),\n",
       " ('Nikmati bersendirian, kami membesarkan keluarga kami sehingga masa lapang tidak lama.',\n",
       "  'Tetapi sukar kerana saya vegan dan rakan-rakan saya tidak'),\n",
       " ('Tiga dan satu dalam perjalanan. Anjing itu seperti kanak-kanak juga.',\n",
       "  'Saya suka anjing mahukan anjing yang serak tetapi belum dapat memilikinya'),\n",
       " ('Adakah anda selalu pergi minum kopi.',\n",
       "  'Setiap pagi sebelum saya pergi ke petugas kebersihan. Ia sukar.'),\n",
       " ('Saya suka dia sedikit.', 'Apa lagi yang anda mahu lakukan?'),\n",
       " ('Dengan keluarga besar kita memerlukan bilik, ditambah dengan anjing.',\n",
       "  'Berapa banyak anak yang anda ada')]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(before[-10:], after[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/wiki-wizard/informations-0.json.translate\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/wiki-wizard/informations-100000.json.translate\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/wiki-wizard/informations-200000.json.translate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "informations-100000.json.translate\n",
      "informations-200000.json.translate\n",
      "informations-0.json.translate\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "files = glob('informations-*.json.translate')\n",
    "\n",
    "for file in files:\n",
    "    print(file)\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "        \n",
    "    for i in data:\n",
    "        if len(i) != 2:\n",
    "            continue\n",
    "        splitted = i[1].split('[EENND]')\n",
    "        if len(splitted) != 3:\n",
    "            continue\n",
    "        l, r = splitted[1:]\n",
    "        l = cleaning(l)\n",
    "        r = cleaning(r)\n",
    "        before.append(l)\n",
    "        after.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('saya dibesarkan di nevada.',\n",
       "  'Nevada adalah yang ke-7 yang paling luas, yang ke-34 paling ramai, tetapi yang ke-9 paling padat penduduknya dari 50 Amerika Syarikat.'),\n",
       " ('saya dibesarkan di nevada.',\n",
       "  'Nevada secara rasmi dikenali sebagai \"Negeri Perak\" kerana pentingnya perak untuk sejarah dan ekonominya.'),\n",
       " ('saya dibesarkan di nevada.',\n",
       "  'Kawasan di selatan Lembangan Besar terletak di Gurun Mojave, sementara Tasik Tahoe dan Sierra Nevada terletak di pinggir barat.'),\n",
       " ('saya dibesarkan di nevada.',\n",
       "  'Nevada bersempadan dengan Oregon di barat laut, Idaho di timur laut, California di sebelah barat, Arizona di tenggara dan Utah di sebelah timur.'),\n",
       " ('nama saya john dan saya tinggal di dallas, tx.',\n",
       "  'Ia adalah bandar paling padat penduduk di Dallas – Fort Worth metroplex, yang merupakan kawasan metropolitan paling ramai keempat di Amerika Syarikat.'),\n",
       " ('saya dibesarkan di nevada.', 'Ibu kota Nevada adalah Carson City.'),\n",
       " ('saya dibesarkan di nevada.',\n",
       "  'Kira-kira 86% tanah negeri diuruskan oleh pelbagai bidang kuasa kerajaan persekutuan A.S., baik awam dan tentera.'),\n",
       " ('saya dibesarkan di nevada.',\n",
       "  'Nevada sebahagian besarnya adalah padang pasir dan separa gersang, kebanyakannya terdapat di dalam Great Basin.'),\n",
       " ('nama saya john dan saya tinggal di dallas, tx.',\n",
       "  'Penduduk bandar ini berada di kedudukan kesembilan di A.S. dan ketiga di Texas selepas Houston dan San Antonio.'),\n",
       " ('nama saya john dan saya tinggal di dallas, tx.',\n",
       "  'Dallas () adalah sebuah bandar di negeri A.S. Texas.')]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(before[-10:], after[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "\n",
    "filename = 'chatbot.tsv'\n",
    "with tf.io.gfile.GFile(filename, 'w') as outfile:\n",
    "        \n",
    "    for i in range(len(before)):\n",
    "        outfile.write('%s\\t%s\\n' % (before[i], after[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from t5.data import preprocessors as prep\n",
    "import functools\n",
    "import t5\n",
    "import gin\n",
    "\n",
    "gin.parse_config_file('pretrained_models_base_operative_config.gin')\n",
    "vocab = 'sp10m.cased.t5.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/t5/models/mesh_transformer.py:210: UserWarning: get_sentencepiece_model_path is deprecated. Please pass the mixture or task vocabulary directly to the Mesh TensorFlow Transformer instead.\n",
      "  \"get_sentencepiece_model_path is deprecated. Please pass the mixture or \"\n"
     ]
    }
   ],
   "source": [
    "def chatbot_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        ['chatbot.tsv']\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "def chatbot_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['chatbot: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('chatbot_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'chatbot_dataset',\n",
    "    dataset_fn = chatbot_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [chatbot_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "325598it [02:29, 2184.75it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "nq_task = t5.data.TaskRegistry.get(\"chatbot_dataset\")\n",
    "ds = nq_task.get_dataset(split='a', sequence_length={\"inputs\": 1024, \"targets\": 1024})\n",
    "results = []\n",
    "for ex in tqdm(tfds.as_numpy(ds)):\n",
    "    results.append((ex['inputs'].tolist(), ex['targets'].tolist()))\n",
    "    \n",
    "with open('chatbot.tsv.parse', 'w') as fopen:\n",
    "    json.dump(results, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([15,\n",
       "  15145,\n",
       "  5746,\n",
       "  50,\n",
       "  709,\n",
       "  2301,\n",
       "  30323,\n",
       "  51,\n",
       "  189,\n",
       "  75,\n",
       "  1773,\n",
       "  3,\n",
       "  1791,\n",
       "  85,\n",
       "  103,\n",
       "  1393,\n",
       "  16727,\n",
       "  13,\n",
       "  17,\n",
       "  353,\n",
       "  189,\n",
       "  18,\n",
       "  192,\n",
       "  3,\n",
       "  15,\n",
       "  3,\n",
       "  1],\n",
       " [1832,\n",
       "  508,\n",
       "  6834,\n",
       "  1481,\n",
       "  16699,\n",
       "  6009,\n",
       "  254,\n",
       "  18,\n",
       "  107,\n",
       "  1393,\n",
       "  16727,\n",
       "  13,\n",
       "  3,\n",
       "  12485,\n",
       "  26,\n",
       "  353,\n",
       "  15358,\n",
       "  1097,\n",
       "  1])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(\"sp10m.cased.t5.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chatbot: Saya suka Pizza lebih baik secara peribadi. Kami mempunyai beberapa pizza yang sangat baik di sini. .'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp.DecodeIds(results[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anda harus mencuba meletakkan hamburger di atas pizza. Rasanya sangat sedap!'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp.DecodeIds(results[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
